R Notebook

lines = readLines(paste0(RESEARCH_HOME, PATH_FILES, NOHUP_FILE))
lines = lines[grepl("\\|[1-6]\\.", lines)]
ff = as_tibble(lines) %>%
  separate(value, into=c("Timestamp", "Title", "ID", "Nodes", "Cores", "Status", "Time", "Stage", "Duration", "Load", "Interval"), sep="\\|") %>%
  separate(ID, into=c(NA, NA, "ID"), sep="-") %>%
  mutate(ID = as.numeric(ID), Stage = str_trim(Stage), Duration = as.numeric(Duration)) %>%
  mutate(uStart = parse_datetime(str_replace(Timestamp, ",", ".")), FFStage = Stage, Status = str_trim(Status)) %>%
  select(uStart, Nodes, Stage, Duration, Interval, FFStage, Status)
head(ff)

stageFF = ff %>% filter(str_trim(Status) == "END") %>% 
  mutate(Stage = paste0(Interval,".",Stage)) %>% 
  select(Nodes, Stage, Duration)
head(stageFF)

p = ggplot(data = stageFF, aes(x = Stage, y = Duration, fill = Nodes)) +
  geom_bar(stat="identity", position=position_dodge(width = 0.75), width = 0.7) + 
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) +
  labs(x="Stages", y="Duration(s)")
ggplotly(p)

stages = rbind(stages0, ff) %>% arrange(uStart)
head(stages, n=100)

n = nrow(stages)
ffstage  = ""
interval = ""
for(row in 1:n){
  if(stages[row, "Status"] == "START"){
    ffstage  = stages[row, "FFStage"]
    interval = stages[row, "Interval"]
  } else {
    if(stages[row, "Status"] == "END"){
      ffstage  = ""
      interval = ""
    } else {
      stages[row, "FFStage"] = ffstage
      stages[row, "Interval"] = interval
    }
  }
}
head(stages, n=100)

s = stages %>% filter(Interval != "") %>% filter(grepl("_", Stage)) %>% 
  mutate(FFStage = paste0(Interval,".",FFStage)) %>% 
  select(Nodes, Stage, Duration, FFStage, Interval)
head(s)

interval = 0
p = ggplot(data = s %>% filter(Interval == interval), aes(x = Stage, y = Duration, fill = Nodes)) +
  geom_bar(stat="identity", position=position_dodge(width = 0.75), width = 0.7) + 
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) +
  labs(title=paste("Time interval", interval), x="Stages", y="Duration(s)")
ggplotly(p)

interval = 1
p = ggplot(data = s %>% filter(Interval == interval), aes(x = Stage, y = Duration, fill = Nodes)) +
  geom_bar(stat="identity", position=position_dodge(width = 0.75), width = 0.7) + 
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) +
  labs(title=paste("Time interval", interval), x="Stages", y="Duration(s)")
ggplotly(p)

interval = 2
p = ggplot(data = s %>% filter(Interval == interval), aes(x = Stage, y = Duration, fill = Nodes)) +
  geom_bar(stat="identity", position=position_dodge(width = 0.75), width = 0.7) + 
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) +
  labs(title=paste("Time interval", interval), x="Stages", y="Duration(s)")
ggplotly(p)

interval = 3
p = ggplot(data = s %>% filter(Interval == interval), aes(x = Stage, y = Duration, fill = Nodes)) +
  geom_bar(stat="identity", position=position_dodge(width = 0.75), width = 0.7) + 
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) +
  labs(title=paste("Time interval", interval), x="Stages", y="Duration(s)")
ggplotly(p)

interval = 4
p = ggplot(data = s %>% filter(Interval == interval), aes(x = Stage, y = Duration, fill = Nodes)) +
  geom_bar(stat="identity", position=position_dodge(width = 0.75), width = 0.7) + 
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) +
  labs(title=paste("Time interval", interval), x="Stages", y="Duration(s)")
ggplotly(p)